#! /usr/bin/perl/ -w
use strict;
use warnings;
require "webPageReader.pl";

my $SUFFIX=".urls";

if(@ARGV<=0){
	die("Please input file path: A file and B file that includes urls inside");
}

my $A_FILE = $ARGV[0];
my $B_FILE = $ARGV[1];
my $A_MINUS_B = $A_FILE."_MINUS_".$B_FILE;
my $B_MINUS_A = $B_FILE."_MINUS_".$A_FILE;
my $A_AND_B = $A_FILE."_AND_".$B_FILE;
my $counting=0;

my $IN_FILE_HANDLER;
my $OUT_FILE_HANDLER;
open(IN_FILE_HANDLER, "<$A_FILE")||die("Can not open file: $A_FILE\n");

my $content="";
my $line;
my %A_Hash=();

print "Begin to craw url in $A_FILE\n";
while($line=<IN_FILE_HANDLER>){
	#remove the \r and \n
	chomp($line);
	print "craw $line\n";
	my $pageContent = getPageContent($line);
	my $linksStr=getAllFollowLinks($pageContent);
	print "get url finished\n";
	sleep(0.5);
	my @linksArray = split("\n", $linksStr);
	for(my $i=0; $i<scalar(@linksArray); $i++){
		$A_Hash{$linksArray[$i]}++;
	}
}


open(IN_FILE_HANDLER2, "<$B_FILE")||die("Can not open file: $B_FILE\n");
my %B_Hash=();
print "Begin to craw url in $B_FILE\n";
$counting=0;
while($line=<IN_FILE_HANDLER2>){
	#remove the \r and \n
	chomp($line);
	print "craw $line\n";
	my $pageContent = getPageContent($line);
	my $linksStr=getAllFollowLinks($pageContent);
	print "get url finished\n";
	sleep(0.5);
	print $counting++;
	my @linksArray = split("\n", $linksStr);
	for(my $i=0; $i<scalar(@linksArray); $i++){
		$B_Hash{$linksArray[$i]}++;
	}
}

open(OUT_FILE_A_B, ">$A_MINUS_B")||die("Can not open file: $A_MINUS_B\n");
open(OUT_FILE_B_A, ">$B_MINUS_A")||die("Can not open file: $B_MINUS_A\n");
open(OUT_FILE_A_AND_B, ">$A_AND_B")||die("Can not open file: $A_AND_B\n");
while(my($k,$v) = each %A_Hash){
	if(exists($B_Hash{$k})){
		print OUT_FILE_A_AND_B $k."\n";
	}
	else{
		print OUT_FILE_A_B $k."\n";
	}
}
close(OUT_FILE_A_B);
close(OUT_FILE_A_AND_B);

open(OUT_FILE_B_A, ">$B_MINUS_A")||die("Can not open file: $B_MINUS_A\n");
while(my($k,$v) = each %B_Hash){
	    if(exists($A_Hash{$k})){
	    	next;
	    }   
	    print OUT_FILE_B_A $k."\n";
}   
close(OUT_FILE_B_A);

